Explore the dataset

In this notebook, we will perform an EDA (Exploratory Data Analysis) on the processed Waymo dataset (data in the processed folder). In the first part, you will create a function to display

In [1]:
from utils import get_dataset
import glob
import matplotlib.pyplot as plt
import matplotlib.patches as patches
import numpy as np
import tensorflow as tf

%matplotlib inline
In [2]:
dataset = get_dataset('/home/workspace/data/train/*.tfrecord')
INFO:tensorflow:Reading unweighted datasets: ['/home/workspace/data/train/*.tfrecord']
INFO:tensorflow:Reading record datasets for input file: ['/home/workspace/data/train/*.tfrecord']
INFO:tensorflow:Number of filenames to read: 86
WARNING:tensorflow:From /data/virtual_envs/sdc-c1-gpu-augment/lib/python3.7/site-packages/object_detection/builders/dataset_builder.py:105: parallel_interleave (from tensorflow.python.data.experimental.ops.interleave_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use `tf.data.Dataset.interleave(map_func, cycle_length, block_length, num_parallel_calls=tf.data.experimental.AUTOTUNE)` instead. If sloppy execution is desired, use `tf.data.Options.experimental_deterministic`.
WARNING:tensorflow:From /data/virtual_envs/sdc-c1-gpu-augment/lib/python3.7/site-packages/object_detection/builders/dataset_builder.py:237: DatasetV1.map_with_legacy_function (from tensorflow.python.data.ops.dataset_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use `tf.data.Dataset.map()
In [3]:
dataset
Out[3]:
<DatasetV1Adapter shapes: {image: (None, None, 3), source_id: (), key: (), filename: (), groundtruth_image_confidences: (None,), groundtruth_verified_neg_classes: (None,), groundtruth_not_exhaustive_classes: (None,), groundtruth_boxes: (None, 4), groundtruth_area: (None,), groundtruth_is_crowd: (None,), groundtruth_difficult: (None,), groundtruth_group_of: (None,), groundtruth_weights: (None,), groundtruth_classes: (None,), groundtruth_image_classes: (None,), original_image_spatial_shape: (2,)}, types: {image: tf.uint8, source_id: tf.string, key: tf.string, filename: tf.string, groundtruth_image_confidences: tf.float32, groundtruth_verified_neg_classes: tf.int64, groundtruth_not_exhaustive_classes: tf.int64, groundtruth_boxes: tf.float32, groundtruth_area: tf.float32, groundtruth_is_crowd: tf.bool, groundtruth_difficult: tf.int64, groundtruth_group_of: tf.bool, groundtruth_weights: tf.float32, groundtruth_classes: tf.int64, groundtruth_image_classes: tf.int64, original_image_spatial_shape: tf.int32}>
In [4]:
for i, element in enumerate(dataset):
    print(element)
    if i >= 2:
        break
{'image': <tf.Tensor: shape=(640, 640, 3), dtype=uint8, numpy=
array([[[73, 85, 65],
        [79, 92, 72],
        [80, 93, 75],
        ...,
        [39, 59, 50],
        [27, 49, 37],
        [32, 54, 41]],

       [[77, 88, 71],
        [77, 90, 72],
        [75, 89, 72],
        ...,
        [38, 58, 49],
        [28, 48, 37],
        [22, 42, 30]],

       [[86, 99, 82],
        [77, 90, 73],
        [70, 84, 69],
        ...,
        [26, 43, 35],
        [28, 45, 35],
        [18, 35, 25]],

       ...,

       [[ 0,  8,  7],
        [ 0,  8,  7],
        [ 0,  9,  8],
        ...,
        [43, 56, 65],
        [45, 58, 67],
        [44, 57, 66]],

       [[ 1, 10,  7],
        [ 0,  9,  6],
        [ 3,  9,  7],
        ...,
        [44, 54, 64],
        [44, 57, 66],
        [39, 52, 61]],

       [[ 0,  8,  5],
        [ 0,  6,  3],
        [ 2,  8,  6],
        ...,
        [44, 54, 64],
        [45, 58, 67],
        [40, 53, 62]]], dtype=uint8)>, 'source_id': <tf.Tensor: shape=(), dtype=string, numpy=b'segment-10599748131695282446_1380_000_1400_000_with_camera_labels_70.tfrecord'>, 'key': <tf.Tensor: shape=(), dtype=string, numpy=b''>, 'filename': <tf.Tensor: shape=(), dtype=string, numpy=b'segment-10599748131695282446_1380_000_1400_000_with_camera_labels_70.tfrecord'>, 'groundtruth_image_confidences': <tf.Tensor: shape=(0,), dtype=float32, numpy=array([], dtype=float32)>, 'groundtruth_verified_neg_classes': <tf.Tensor: shape=(0,), dtype=int64, numpy=array([], dtype=int64)>, 'groundtruth_not_exhaustive_classes': <tf.Tensor: shape=(0,), dtype=int64, numpy=array([], dtype=int64)>, 'groundtruth_boxes': <tf.Tensor: shape=(37, 4), dtype=float32, numpy=
array([[0.45444873, 0.00657906, 0.48898882, 0.07105388],
       [0.61697304, 0.        , 0.97779614, 0.25074333],
       [0.517213  , 0.78537565, 0.5509638 , 0.8022838 ],
       [0.47838008, 0.26151773, 0.5380851 , 0.27204424],
       [0.45506558, 0.15633464, 0.47850338, 0.16899931],
       [0.5617697 , 0.1600357 , 0.76358247, 0.35839444],
       [0.46900493, 0.27911672, 0.51292014, 0.29063007],
       [0.46604434, 0.26694545, 0.51292014, 0.27845883],
       [0.4653042 , 0.21793145, 0.54227924, 0.23405015],
       [0.5367693 , 0.7588401 , 0.60173756, 0.815749  ],
       [0.5188413 , 0.7944218 , 0.54894054, 0.806922  ],
       [0.50167406, 0.6512448 , 0.53107405, 0.6566725 ],
       [0.47097865, 0.13059439, 0.5168676 , 0.21809593],
       [0.50218827, 0.6736963 , 0.5414163 , 0.68520963],
       [0.5119333 , 0.7825795 , 0.55229604, 0.7954746 ],
       [0.54458183, 0.76925725, 0.6100438 , 0.84699976],
       [0.48491803, 0.3327357 , 0.5320407 , 0.3830663 ],
       [0.5215552 , 0.8256723 , 0.53438437, 0.8302777 ],
       [0.4988574 , 0.6009974 , 0.5333975 , 0.6240241 ],
       [0.53561795, 0.74760115, 0.5943362 , 0.7890492 ],
       [0.52426904, 0.76892793, 0.5435128 , 0.78636247],
       [0.55658865, 0.8080739 , 0.6138269 , 0.8607065 ],
       [0.53759164, 0.26414937, 0.67723227, 0.3832304 ],
       [0.56887513, 0.8558047 , 0.64575154, 0.90817404],
       [0.4768998 , 0.2962223 , 0.513907  , 0.30773565],
       [0.34133017, 0.3789539 , 0.72941273, 0.6072474 ],
       [0.5144005 , 0.68887484, 0.563673  , 0.7811697 ],
       [0.45950645, 0.04597219, 0.50860256, 0.1313345 ],
       [0.50083107, 0.6408008 , 0.5383318 , 0.6493535 ],
       [0.5083736 , 0.7679412 , 0.5368516 , 0.7948684 ],
       [0.48504138, 0.22023411, 0.564977  , 0.24885304],
       [0.55406004, 0.89639777, 0.6908636 , 1.        ],
       [0.45099473, 0.        , 0.49244282, 0.01578975],
       [0.48282093, 0.32445708, 0.54515785, 0.3444137 ],
       [0.5359264 , 0.8060177 , 0.6251139 , 0.88924295],
       [0.5026818 , 0.65247846, 0.532041  , 0.66004455],
       [0.49696594, 0.6588931 , 0.52706516, 0.666788  ]], dtype=float32)>, 'groundtruth_area': <tf.Tensor: shape=(0,), dtype=float32, numpy=array([], dtype=float32)>, 'groundtruth_is_crowd': <tf.Tensor: shape=(0,), dtype=bool, numpy=array([], dtype=bool)>, 'groundtruth_difficult': <tf.Tensor: shape=(0,), dtype=int64, numpy=array([], dtype=int64)>, 'groundtruth_group_of': <tf.Tensor: shape=(0,), dtype=bool, numpy=array([], dtype=bool)>, 'groundtruth_weights': <tf.Tensor: shape=(37,), dtype=float32, numpy=
array([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,
       1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,
       1., 1., 1.], dtype=float32)>, 'groundtruth_classes': <tf.Tensor: shape=(37,), dtype=int64, numpy=
array([1, 1, 2, 2, 2, 1, 2, 2, 2, 1, 2, 2, 1, 2, 2, 1, 1, 2, 1, 1, 1, 1,
       1, 1, 2, 1, 1, 1, 2, 1, 2, 1, 1, 2, 1, 2, 2])>, 'groundtruth_image_classes': <tf.Tensor: shape=(0,), dtype=int64, numpy=array([], dtype=int64)>, 'original_image_spatial_shape': <tf.Tensor: shape=(2,), dtype=int32, numpy=array([640, 640], dtype=int32)>}
{'image': <tf.Tensor: shape=(640, 640, 3), dtype=uint8, numpy=
array([[[ 27,  32,  36],
        [ 36,  39,  44],
        [ 41,  40,  45],
        ...,
        [140, 148, 161],
        [161, 171, 183],
        [160, 172, 184]],

       [[ 30,  35,  39],
        [ 37,  40,  45],
        [ 42,  41,  46],
        ...,
        [152, 160, 173],
        [173, 183, 195],
        [161, 173, 185]],

       [[ 34,  39,  43],
        [ 40,  43,  48],
        [ 45,  44,  49],
        ...,
        [148, 156, 169],
        [166, 176, 188],
        [157, 169, 181]],

       ...,

       [[  1,  10,   9],
        [  0,   8,   7],
        [  0,   7,   6],
        ...,
        [ 94,  70,  84],
        [ 94,  70,  84],
        [ 94,  70,  84]],

       [[  1,   9,  11],
        [  0,   7,   9],
        [  0,   6,   8],
        ...,
        [ 95,  71,  84],
        [ 95,  71,  85],
        [ 94,  70,  84]],

       [[  1,   9,  11],
        [  0,   7,   9],
        [  0,   6,   8],
        ...,
        [ 96,  72,  85],
        [ 95,  71,  85],
        [ 94,  70,  84]]], dtype=uint8)>, 'source_id': <tf.Tensor: shape=(), dtype=string, numpy=b'segment-11847506886204460250_1640_000_1660_000_with_camera_labels_40.tfrecord'>, 'key': <tf.Tensor: shape=(), dtype=string, numpy=b''>, 'filename': <tf.Tensor: shape=(), dtype=string, numpy=b'segment-11847506886204460250_1640_000_1660_000_with_camera_labels_40.tfrecord'>, 'groundtruth_image_confidences': <tf.Tensor: shape=(0,), dtype=float32, numpy=array([], dtype=float32)>, 'groundtruth_verified_neg_classes': <tf.Tensor: shape=(0,), dtype=int64, numpy=array([], dtype=int64)>, 'groundtruth_not_exhaustive_classes': <tf.Tensor: shape=(0,), dtype=int64, numpy=array([], dtype=int64)>, 'groundtruth_boxes': <tf.Tensor: shape=(43, 4), dtype=float32, numpy=
array([[0.474556  , 0.46111006, 0.49157932, 0.4773932 ],
       [0.5447464 , 0.15529877, 0.72731537, 0.29905128],
       [0.48208082, 0.9028118 , 0.5639901 , 0.999853  ],
       [0.48454794, 0.66777486, 0.54869384, 0.6835646 ],
       [0.52170324, 0.14832497, 0.55574983, 0.19839163],
       [0.4956501 , 0.7944218 , 0.54449964, 0.8707389 ],
       [0.47867966, 0.31351584, 0.49337682, 0.33621362],
       [0.5079859 , 0.44589597, 0.6032178 , 0.54293716],
       [0.48134065, 0.8567584 , 0.5617697 , 1.        ],
       [0.5338909 , 0.70001227, 0.6391559 , 0.80571586],
       [0.4719655 , 0.45148817, 0.48232752, 0.46036988],
       [0.49268955, 0.17006877, 0.5504208 , 0.26020193],
       [0.39992476, 0.        , 0.9933233 , 0.20154957],
       [0.59226364, 0.3492824 , 0.9484212 , 0.6217872 ],
       [0.49244282, 0.93472034, 0.5881682 , 0.9569247 ],
       [0.47418594, 0.53372645, 0.48898882, 0.54129237],
       [0.4773153 , 0.39775628, 0.4924558 , 0.40883678],
       [0.46925163, 0.43323126, 0.48504138, 0.43915242],
       [0.48400518, 0.24474113, 0.6366723 , 0.3776382 ],
       [0.48282096, 0.51744324, 0.5721317 , 0.6641564 ],
       [0.50539535, 0.79746455, 0.57472223, 0.9496054 ],
       [0.47270563, 0.4638239 , 0.48158738, 0.4717188 ],
       [0.47591293, 0.22631975, 0.49416983, 0.2421095 ],
       [0.6306525 , 0.94843763, 0.80522794, 1.        ],
       [0.48158738, 0.24013577, 0.49639025, 0.28882083],
       [0.48306766, 0.74014455, 0.5324106 , 0.7552764 ],
       [0.47788665, 0.38355935, 0.49713042, 0.3993491 ],
       [0.46925163, 0.42582983, 0.47615966, 0.4287904 ],
       [0.47936693, 0.46481076, 0.50453186, 0.48652166],
       [0.50576544, 0.8727126 , 0.5551084 , 0.8885024 ],
       [0.4801071 , 0.49096254, 0.5037917 , 0.49852845],
       [0.5037917 , 0.9010026 , 0.56102955, 0.9115291 ],
       [0.49416983, 0.8843905 , 0.55387485, 0.9024829 ],
       [0.47319907, 0.25641897, 0.5037917 , 0.27516928],
       [0.4773932 , 0.54326606, 0.4887421 , 0.5501741 ],
       [0.48232752, 0.21760249, 0.49268955, 0.24030025],
       [0.47788665, 0.35197985, 0.5030516 , 0.37369075],
       [0.48578152, 0.20016798, 0.49811727, 0.20378646],
       [0.5505195 , 0.75896066, 0.7209501 , 0.95554304],
       [0.47603628, 0.40749067, 0.49009904, 0.41883957],
       [0.47122535, 0.4200592 , 0.48602825, 0.43121573],
       [0.5422299 , 0.38405278, 0.6966734 , 0.529779  ],
       [0.4786268 , 0.39392138, 0.49639025, 0.4028031 ]], dtype=float32)>, 'groundtruth_area': <tf.Tensor: shape=(0,), dtype=float32, numpy=array([], dtype=float32)>, 'groundtruth_is_crowd': <tf.Tensor: shape=(0,), dtype=bool, numpy=array([], dtype=bool)>, 'groundtruth_difficult': <tf.Tensor: shape=(0,), dtype=int64, numpy=array([], dtype=int64)>, 'groundtruth_group_of': <tf.Tensor: shape=(0,), dtype=bool, numpy=array([], dtype=bool)>, 'groundtruth_weights': <tf.Tensor: shape=(43,), dtype=float32, numpy=
array([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,
       1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,
       1., 1., 1., 1., 1., 1., 1., 1., 1.], dtype=float32)>, 'groundtruth_classes': <tf.Tensor: shape=(43,), dtype=int64, numpy=
array([1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 2, 1, 1, 1, 1,
       2, 1, 1, 2, 1, 2, 1, 2, 2, 2, 2, 2, 2, 1, 1, 2, 1, 1, 1, 1, 1])>, 'groundtruth_image_classes': <tf.Tensor: shape=(0,), dtype=int64, numpy=array([], dtype=int64)>, 'original_image_spatial_shape': <tf.Tensor: shape=(2,), dtype=int32, numpy=array([640, 640], dtype=int32)>}
{'image': <tf.Tensor: shape=(640, 640, 3), dtype=uint8, numpy=
array([[[10, 20, 11],
        [11, 21, 12],
        [20, 27, 20],
        ...,
        [50, 60, 61],
        [38, 45, 51],
        [33, 40, 46]],

       [[18, 28, 20],
        [19, 29, 21],
        [18, 25, 18],
        ...,
        [42, 52, 54],
        [38, 47, 52],
        [42, 51, 56]],

       [[21, 32, 26],
        [20, 31, 25],
        [12, 21, 16],
        ...,
        [40, 51, 53],
        [36, 47, 51],
        [46, 57, 63]],

       ...,

       [[59, 60, 44],
        [58, 59, 43],
        [65, 63, 48],
        ...,
        [ 4, 12, 14],
        [ 5, 13, 16],
        [ 4, 14, 16]],

       [[66, 69, 50],
        [59, 60, 42],
        [67, 66, 48],
        ...,
        [ 6, 14, 16],
        [ 6, 14, 17],
        [ 5, 13, 16]],

       [[67, 70, 51],
        [59, 62, 43],
        [65, 64, 46],
        ...,
        [10, 15, 18],
        [ 7, 15, 18],
        [ 6, 14, 17]]], dtype=uint8)>, 'source_id': <tf.Tensor: shape=(), dtype=string, numpy=b'segment-1146261869236413282_1680_000_1700_000_with_camera_labels_150.tfrecord'>, 'key': <tf.Tensor: shape=(), dtype=string, numpy=b''>, 'filename': <tf.Tensor: shape=(), dtype=string, numpy=b'segment-1146261869236413282_1680_000_1700_000_with_camera_labels_150.tfrecord'>, 'groundtruth_image_confidences': <tf.Tensor: shape=(0,), dtype=float32, numpy=array([], dtype=float32)>, 'groundtruth_verified_neg_classes': <tf.Tensor: shape=(0,), dtype=int64, numpy=array([], dtype=int64)>, 'groundtruth_not_exhaustive_classes': <tf.Tensor: shape=(0,), dtype=int64, numpy=array([], dtype=int64)>, 'groundtruth_boxes': <tf.Tensor: shape=(33, 4), dtype=float32, numpy=
array([[0.5097129 , 0.60017496, 0.6370177 , 0.62484646],
       [0.46974507, 0.5763259 , 0.54277265, 0.6618537 ],
       [0.5565887 , 0.11924551, 0.67895925, 0.16760162],
       [0.46357718, 0.8689297 , 0.5351245 , 0.99689245],
       [0.5025581 , 0.5113576 , 0.6007506 , 0.530108  ],
       [0.40917617, 0.41966152, 0.42225233, 0.42936546],
       [0.40313205, 0.44392207, 0.41398752, 0.45115903],
       [0.40855977, 0.42805028, 0.41941524, 0.43314898],
       [0.4670312 , 0.02352015, 0.5869346 , 0.22122097],
       [0.4505013 , 0.27336004, 0.49589685, 0.2898077 ],
       [0.43989256, 0.3376704 , 0.48972896, 0.35049954],
       [0.40855977, 0.41472745, 0.41941524, 0.42245767],
       [0.4650575 , 0.29803154, 0.5395654 , 0.43224442],
       [0.39858988, 0.57516867, 0.4611585 , 0.6028007 ],
       [0.6142795 , 0.8154775 , 1.        , 1.        ],
       [0.43175098, 0.3396441 , 0.48800197, 0.37944743],
       [0.43175098, 0.36299977, 0.4702385 , 0.3916187 ],
       [0.44087943, 0.29227486, 0.50551873, 0.3445784 ],
       [0.4474175 , 0.43018782, 0.50046396, 0.44548452],
       [0.41176707, 0.5965565 , 0.4655509 , 0.64359677],
       [0.44507366, 0.42533562, 0.48849738, 0.4381648 ],
       [0.49762383, 0.5134958 , 0.6007506 , 0.54507536],
       [0.5939245 , 0.41897658, 0.9097195 , 0.6716124 ],
       [0.47615966, 0.41530332, 0.5699113 , 0.57813513],
       [0.4551889 , 0.48586378, 0.5405522 , 0.5944183 ],
       [0.3970759 , 0.50942427, 0.44702402, 0.5392783 ],
       [0.5032983 , 0.5118511 , 0.6000105 , 0.5296145 ],
       [0.4319977 , 0.2926038 , 0.45124146, 0.31464365],
       [0.4902224 , 0.79853374, 0.5963098 , 1.        ],
       [0.43249112, 0.3151371 , 0.44778743, 0.34013754],
       [0.41084194, 0.31842667, 0.42206737, 0.32862422],
       [0.445567  , 0.27226335, 0.4870151 , 0.30318502],
       [0.513907  , 0.3819146 , 0.69104826, 0.4253364 ]], dtype=float32)>, 'groundtruth_area': <tf.Tensor: shape=(0,), dtype=float32, numpy=array([], dtype=float32)>, 'groundtruth_is_crowd': <tf.Tensor: shape=(0,), dtype=bool, numpy=array([], dtype=bool)>, 'groundtruth_difficult': <tf.Tensor: shape=(0,), dtype=int64, numpy=array([], dtype=int64)>, 'groundtruth_group_of': <tf.Tensor: shape=(0,), dtype=bool, numpy=array([], dtype=bool)>, 'groundtruth_weights': <tf.Tensor: shape=(33,), dtype=float32, numpy=
array([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,
       1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.],
      dtype=float32)>, 'groundtruth_classes': <tf.Tensor: shape=(33,), dtype=int64, numpy=
array([2, 1, 2, 1, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 2,
       1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 2])>, 'groundtruth_image_classes': <tf.Tensor: shape=(0,), dtype=int64, numpy=array([], dtype=int64)>, 'original_image_spatial_shape': <tf.Tensor: shape=(2,), dtype=int32, numpy=array([640, 640], dtype=int32)>}

Write a function to display an image and the bounding boxes

Implement the display_images function below. This function takes a batch as an input and display an image with its corresponding bounding boxes. The only requirement is that the classes should be color coded (eg, vehicles in red, pedestrians in blue, cyclist in green).

In [21]:
def display_images(batch):
    '''displays image in batch along with correctly color-coded bounding boxes 
    '''
    bbox_colors = {
        1: 'red',  # cars
        2: 'blue', # pedestrians
        4: 'green' # cyclists
    }
    
    fig, ax = plt.subplots(figsize=(15, 15))
    image = batch['image'].numpy()
    ground_truth_boxes = batch['groundtruth_boxes'].numpy()
    ground_truth_labels = batch['groundtruth_classes'].numpy()
    
    for bbox, label in zip(ground_truth_boxes, ground_truth_labels):
        anchor = (bbox[1] * image.shape[1], bbox[0] * image.shape[0])
        width = (bbox[3] - bbox[1]) * image.shape[1]
        height = (bbox[2] - bbox[0]) * image.shape[0]
        box_patch = patches.Rectangle(anchor,
                                      width,
                                      height,
                                      linewidth=1,
                                      facecolor='None',
                                      edgecolor=bbox_colors[label]
                                     )
        ax.add_patch(box_patch)

    # remove tick labels 
    ax.set_xticks([])
    ax.set_yticks([])    
    ax.imshow(image)

Display 10 images

Using the dataset created in the second cell and the function you just coded, display 10 random images with the associated bounding boxes. You can use the methods take and shuffle on the dataset.

In [22]:
# Display 10 random images in dataset
for batch in dataset.shuffle(50).take(10):
    display_images(batch)

Additional EDA

In this last part, you are free to perform any additional analysis of the dataset. What else would like to know about the data? For example, think about data distribution. So far, you have only looked at a single file...

In [7]:
# How many objects of each type over all images?
object_count_dict = {
    1: 0,   # cars
    2: 0,   # pedestrians
    4: 0    # cyclists
}

object_names_dict = {
    1: 'cars',
    2: 'pedestrians',
    4: 'cyclists'
}

# How many images have which class(es) in them?
classes_count_dict = {
    0: 0,   # no objects
    1: 0,   # just cars
    2: 0,   # just pedestrians
    3: 0,   # no cyclists
    4: 0,   # just cyclists
    5: 0,   # no pedestrians
    6: 0,   # no cars
    7: 0,   # all of the three
}

classes_names_dict = {
    0: 'no objects',
    1: 'just cars',
    2: 'just pedestrians',
    3: 'no cyclists',
    4: 'just cyclists',
    5: 'no pedestrians',
    6: 'no cars',
    7: 'all three'
}

sample_size = 50000

for batch in dataset.take(sample_size):
    this_count = {1: 0, 2: 0, 4: 0}
    classes = 0
    for label in batch['groundtruth_classes'].numpy():
        this_count[label] += 1
        object_count_dict[label] += 1        
    for key, val in this_count.items():
        if val:
            classes += key
    classes_count_dict[classes] += 1
    
print(classes_count_dict)
{0: 1253, 1: 22203, 2: 491, 3: 21312, 4: 116, 5: 1243, 6: 57, 7: 3325}
In [8]:
bars = list(classes_names_dict.values())
plt.bar(bars, classes_count_dict.values())
plt.xticks(bars, bars, rotation=90)
plt.xlabel('objects in image')
plt.ylabel('number of images')
plt.title(f'Distribution of objects in {sample_size} images out of the dataset')
plt.show()
In [9]:
bars = list(object_names_dict.values())
plt.bar(bars, object_count_dict.values())
plt.xticks(bars, bars, rotation=90)
plt.xlabel('objects in image')
plt.ylabel('number of images')
plt.title(f'Number of total objects in {sample_size} images out of the dataset')
plt.show()
In [ ]: